{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.utils import io\n",
    "import sys\n",
    "import pandas as pd\n",
    "import bs4 as bs\n",
    "import urllib2\n",
    "import re\n",
    "import numpy as np\n",
    "import uuid\n",
    "from pdfminer.pdfparser import PDFParser\n",
    "from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter\n",
    "from pdfminer.layout import LAParams\n",
    "from pdfminer.image import ImageWriter\n",
    "from pdfminer.converter import  TextConverter\n",
    "from pdfminer.pdfpage import PDFPage\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf-8')\n",
    "import time\n",
    "from selenium import webdriver\n",
    "import glob\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# combine all search urls here with their search result count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "url_list=[\n",
    "    'http://www.biorxiv.org/search/deep%252Blearning%252Bmedical%252Bimaging?page=',\n",
    "    'http://www.biorxiv.org/search/deep%252Blearning%252Bradiology?page=',\n",
    "    'http://www.biorxiv.org/search/deep%252Blearning%252Bpathology?page=',\n",
    "    'http://www.biorxiv.org/search/deep%252Blearning%252Bgenomics?page=',\n",
    "    'http://www.biorxiv.org/search/convolutional%252Bneural%252Bnetwork?page=',\n",
    "    'http://www.biorxiv.org/search/autoencoder?page=',\n",
    "    'http://www.biorxiv.org/search/deep%252Blearning?page=',\n",
    "    'http://www.biorxiv.org/search/cnn?page='\n",
    "]\n",
    "\n",
    "result_list =[\n",
    "    319,\n",
    "    52,\n",
    "    238,\n",
    "    730,\n",
    "    342,\n",
    "    68,\n",
    "    1192,\n",
    "    136\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-a09a7039ea2e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0murls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msoup\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfindAll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"a\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m \u001b[0;34m\"class\"\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0;34m\"highwire-cite-linked-title\"\u001b[0m \u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m             \u001b[0mall_urls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextend\u001b[0m \u001b[0;34m(\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'http://www.biorxiv.org'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murls\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'href'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m]\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m             \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m             \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"biorxiv_temp_urls.npy\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_urls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0murllib2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mHTTPError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# go through above list and get all individual urls to papers\n",
    "all_urls = []\n",
    "for url_1,max_result in iter(zip(url_list,result_list)):\n",
    "    page_number_list = np.arange(0,np.ceil(max_result/10.0))\n",
    "    print page_number_list\n",
    "    for i in page_number_list:\n",
    "        url = url_1 + str(int(i))\n",
    "        print url\n",
    "        try:\n",
    "            # fetch url\n",
    "            response = urllib2.urlopen(url)\n",
    "            # convert to bs\n",
    "            soup = bs.BeautifulSoup(response,\"html\")\n",
    "            urls = soup.findAll(\"a\", { \"class\" : \"highwire-cite-linked-title\" })\n",
    "            all_urls.extend ( ['http://www.biorxiv.org' + str(urls[x]['href']) for x in range(len(urls)) ] )\n",
    "            time.sleep(2)\n",
    "            np.save(\"biorxiv_temp_urls.npy\", all_urls)\n",
    "        except urllib2.HTTPError:\n",
    "            print str(i) + \" no http\"\n",
    "            pass\n",
    "\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# load back file and remove duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "urls = np.load('/home/ahmed/Dropbox/DFCI/14_zoo/scrap-science/biorxiv/biorxiv_temp_urls.npy')\n",
    "print urls.shape\n",
    "urls = list(set(urls))\n",
    "print len(urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'http://www.biorxiv.org/content/early/2017/04/07/125229'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urls[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# now lets scrap\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# now go through all these urls and scrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class redirect_output(object):\n",
    "    \"\"\"context manager for reditrecting stdout/err to files\"\"\"\n",
    "\n",
    "\n",
    "    def __init__(self, stdout='', stderr=''):\n",
    "        self.stdout = stdout\n",
    "        self.stderr = stderr\n",
    "\n",
    "    def __enter__(self):\n",
    "        self.sys_stdout = sys.stdout\n",
    "        self.sys_stderr = sys.stderr\n",
    "\n",
    "        if self.stdout:\n",
    "            sys.stdout = open(self.stdout, 'w')\n",
    "        if self.stderr:\n",
    "            if self.stderr == self.stdout:\n",
    "                sys.stderr = sys.stdout\n",
    "            else:\n",
    "                sys.stderr = open(self.stderr, 'w')\n",
    "\n",
    "    def __exit__(self, exc_type, exc_value, traceback):\n",
    "        sys.stdout = self.sys_stdout\n",
    "        sys.stderr = self.sys_stderr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# search result page\n",
    "allTitles = []\n",
    "allAuthors = []\n",
    "allFullURLs = []\n",
    "# abstract page\n",
    "allEmails = [] \n",
    "allYears = []\n",
    "allUses = []\n",
    "allAbstracts = []\n",
    "allComments = [] # combine with year into shortDetails\n",
    "# extra one\n",
    "allPdfURLs = []\n",
    "#added at pandas level\n",
    "# source = arvix\n",
    "# key = uuid\n",
    "\n",
    "total = 0\n",
    "print total\n",
    "fileToSave = \"temp_biorxiv_13.csv\"\n",
    "# loop through the main search result pages\n",
    "\n",
    "for idx,url in enumerate(urls[total:]):\n",
    "\n",
    "    try:\n",
    "        print url, idx\n",
    "        # fetch url\n",
    "        response = urllib2.urlopen(url)\n",
    "        # convert to bs\n",
    "        soup = bs.BeautifulSoup(response,\"html\")\n",
    "\n",
    "        # 1 get title\n",
    "        titles = soup.findAll(\"h1\", { \"class\" : \"highwire-cite-title\" })\n",
    "        allTitles.append( titles[0].text )\n",
    "\n",
    "        # 2 get authors\n",
    "        authors = soup.findAll(\"span\", { \"class\" : \"highwire-citation-authors\" })\n",
    "        allAuthors.append( authors[0].text )\n",
    "\n",
    "        # 3 get all fullURLs\n",
    "        allFullURLs.append ( url )\n",
    "\n",
    "        # 4 get emails through pdf\n",
    "        pdfUrl = url + '.full.pdf'\n",
    "        print pdfUrl\n",
    "        allPdfURLs.append(pdfUrl)\n",
    "        allEmails.append( download_pdf(pdfUrl, \n",
    "                                       url, 'pdf'\n",
    "                                       , '/home/ahmed/Dropbox/DFCI/08_radiomics.io/chromedriver') )\n",
    "\n",
    "        # 5 get year\n",
    "        years = soup.findAll(\"li\", { \"class\" : \"published\" })\n",
    "        tempList1 = [ int( str(years[x].text).strip()\n",
    "                          [str(years[x].text).strip().find('201')\n",
    "                           :\n",
    "                           (str(years[x].text).strip().find('201')+4)] )\n",
    "                          for x in range(len(years)) ]\n",
    "        allYears.append(tempList1[0])\n",
    "\n",
    "        # 6 get use based on year\n",
    "        if tempList1[0] >= 2014:\n",
    "            allUses.append(1)\n",
    "        else:\n",
    "            allUses.append( 0 )  \n",
    "\n",
    "        # 7 get abstracts\n",
    "        abstracts = soup.find_all(\"p\" , {'id' : 'p-2' })\n",
    "        if len(abstracts) == 0:\n",
    "            allAbstracts.append(\"\")\n",
    "        else:\n",
    "            allAbstracts.append( str(abstracts[0].text).strip() )  \n",
    "\n",
    "        # 8 get comments\n",
    "        # used when javascript rpoduces the tag and bs4 cant find it\n",
    "        driver = webdriver.Chrome('/home/ahmed/Dropbox/DFCI/08_radiomics.io/chromedriver')\n",
    "        driver.get(url)\n",
    "        siko = driver.find_element_by_class_name(\"pub_jnl\")\n",
    "\n",
    "        if 'article is a preprint and has not been peer-reviewed' in siko.text.strip():\n",
    "            allComments.append(\"Retrieved from bioRxiv\")\n",
    "        else:\n",
    "            allComments.append( str(siko.text.strip()) [:str(siko.text.strip()).find(' doi:')] )\n",
    "\n",
    "        driver.close()\n",
    "\n",
    "\n",
    "        ######\n",
    "        df = pd.DataFrame()\n",
    "        # those that match pubmed\n",
    "        df['Title'] = allTitles\n",
    "        df['Description'] = allAuthors\n",
    "        df['ShortDetails'] = [allComments[x] + \". \" + str(allYears[x]) for x in range(len(allYears))  ]\n",
    "        df['abstract'] =   allAbstracts\n",
    "        df['email'] =   allEmails\n",
    "        df['fullURL'] = allFullURLs                                                                           \n",
    "        df['source'] = ['biorxiv' for x in range(len(allYears))]                                                                            \n",
    "        df['year'] = allYears\n",
    "        df['key'] = [str(uuid.uuid4()) for x in range(len(allYears))]\n",
    "        df['use'] = allUses\n",
    "        # extra one\n",
    "        df['pdfURL']= allPdfURLs\n",
    "\n",
    "        df.to_csv( fileToSave)\n",
    "\n",
    "    except urllib2.HTTPError:\n",
    "        print str(i) + \" no http\"\n",
    "        pass\n",
    "\n",
    "\n",
    "    print \"done\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame()\n",
    "# those that match pubmed\n",
    "df['Title'] = allTitles\n",
    "df['Description'] = allAuthors\n",
    "df['ShortDetails'] = [allComments[x] + \". \" + str(allYears[x]) for x in range(len(allYears))  ]\n",
    "df['abstract'] =   allAbstracts\n",
    "df['email'] =   allEmails\n",
    "df['fullURL'] = allFullURLs                                                                           \n",
    "df['source'] = ['biorxiv' for x in range(len(allYears))]                                                                            \n",
    "df['year'] = allYears\n",
    "df['key'] = [str(uuid.uuid4()) for x in range(len(allYears))]\n",
    "df['use'] = allUses\n",
    "# extra one\n",
    "df['pdfURL']= allPdfURLs\n",
    "    \n",
    "    \n",
    "df.head(n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lista = [allTitles,allAuthors,allFullURLs,allComments,allYears,allAbstracts,allEmails,allUses,allPdfURLs]\n",
    "for lis in iter(lista):\n",
    "    print len(lis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv( \"test.csv\")\n",
    "# then need to add\n",
    "# var Data1 = ....\n",
    "# export default Data1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "venvradiomicsio",
   "language": "python",
   "name": "venvradiomicsio"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
